# frontend.py
# 
# frontend for hp project

import os, random, time, smtplib
from operator import itemgetter
import urllib2
from xml.dom import minidom


### options ###
gid = 0                             # group id pool
sleeptime = 5                       # how many seconds between job checks
inboxdir  = 'inbox'                 # working directory for incoming job requests
jobfile   = 'requests'              # name of file with requests
outboxdir = 'outbox'                # working directory for outgoing job data

### config for job request email ###
emailuser = 'csi702job'             # gmail user name
emailpass = 'amhsamhs'              # gmail password

# retrieves email feed from gmail
def getEmailFeed(user, passwd):
    auth_handler = urllib2.HTTPBasicAuthHandler()
    auth_handler.add_password(realm='New mail feed',uri='https://mail.google.com',user='%s@gmail.com' % user,passwd=passwd)
    opener = urllib2.build_opener(auth_handler)
    urllib2.install_opener(opener)
    feed = urllib2.urlopen('https://mail.google.com/mail/feed/atom')
    return feed.read()
    
# extracts text from xml element
def getText(nodelist):
    s = ""
    for node in nodelist:
        if node.nodeType == node.TEXT_NODE:
            s = s + node.data
    return str(s)
    
# returns whether lock is locked
def isLocked():
    return os.path.exists('connect.lock')
    
# unlocks or locks lock, request: 'lock' or 'unlock'
def changeLock(request):
    if request == 'lock' and not isLocked():
        open('connect.lock', 'w').close()
        #print 'lock added'
    elif request == 'unlock' and isLocked():
        os.remove('connect.lock')
        #print 'lock removed'
    else:
        #print 'your lock request doesn\'t make sense'
        pass
        
    return

# reads current pool of workers from connect.data
def updatePool():
    global pool
    pool = {}
    done = False
    # read current c.d
    cfile = open('connect.data', 'r')
    print 'Getting current connection data...', 
    for line in cfile:
        if line.find('---') == -1:
            # populate pool with data
            x = line.split()
            ### make wid based on ip
            wid = x[0]+':'+x[1]
            pool[wid] = {}
            pool[wid]['ip']     = x[0]
            pool[wid]['port']   = x[1]
            pool[wid]['status'] = x[2]
            pool[wid]['gid']    = int(x[3])
            pool[wid]['master'] = int(x[4]) 
        
    cfile.close()
    print 'done:', availableInPool(), 'of', len(pool), 'available.'
    
    return
    
# returns number of idle workers in pool
def availableInPool():
    global pool
    num = 0
    for worker in pool.keys():
        if pool[worker]['status']=='i':
            num += 1
    return num
# returns group of workers marked (f)inished
def getFinished():
    global pool
    group = {}
    for key in pool:
        if pool[key]['status'] == 'f':
            group[key] = pool[key]
            
    return group
    
# tries to return a group (dict) of n workers from current pool, returns None if it fails
def recruitGroup(n, job):
    global gid
    global pool
    global jobowners
    global jobs
    group = {}
    k = 0
    a = availableInPool()
    # check if there are enough workers available
    if n > a:
        print 'Not enough workers in pool, need', n-a, 'more...'
        return None
        
    # iterate over pool, choosing workers, preferring similar ip's
    for wid in sorted(pool, key=lambda key: pool[key]['ip']):
        if k < n and pool[wid]['status']=='i':
            group[wid] = pool[wid]
            k += 1
    
    
    
    # randomly assign a master from the group
    master = random.choice(group.keys())
    group[master]['master'] = 1 
    
    # add contact email to jobowners
    jobowners[gid] = job['email']
    
    # add job to jobs dict
    jobs[gid] = job
    
    # set group members to newbusy('n') and give them gid
    for member in group.keys():
        group[member]['status'] = 'n'
        group[member]['gid']    = gid
        
    gid += 1
    return group

# checks for a new job, returns it (dict), None if failure
# method can be 'file' or 'email'
def popJob():
    global inboxdir
    global jobowners
    job = {}
    req  = open(jobfile, 'r')
    file = req.readlines()
    if len(file)==0:
        #print 'No new jobs, returning...'
        return None
    print 'Job available...'
    file.reverse()                          # reverse lines of file, to treat it as a stack
    j = file.pop()                          # this is the oldest job
    req.close()                             # close file 
    req = open(jobfile, 'w')                # open it again for writing
    file.reverse()                          # put the lines back in normal order
    req.writelines(file)                    # write them back to the file (less one job)
    
    j = j.split(',')
    job['n']     = int(j[0])
    job['exec']  = j[1]
    job['out']   = j[2]
    job['email'] = j[3].strip('\n')
    
    print job
    return job
    

# puts a job on the job stack
def pushJob(job):
    # append the job to end of request file
    # putting it at end allows a potentially smaller job to be considered at next run
    
    # open request file for append
    f = open(jobfile, 'a')
    # append job to request file
    s = str(job['n'])+','+str(job['exec'])+','+str(job['out']+','+str(job['email'])+'\n')
    f.write(s)
    f.close()
    
    return None
# creates the appropriate files for a given job assigned to a certain group
def generateJobFiles(group, job):
    global outboxdir
    global inboxdir
    global gid
    g = str(gid - 1)    # current gid is one higher than last used gid
    
    # mk dir for gid
    os.mkdir(outboxdir+'/'+g)
    
    # name for executable: codefileX (X is gid)
    execfn = 'codefile'+g
    # mv execfile from inbox to outbox with new name
    os.rename(inboxdir+'/'+job['exec'], outboxdir+'/'+g+'/'+execfn)
    
    # name for hostfile:   hostfileX (X is gid)
    hostfn = 'hostfile'+g
    # open hostfile for writing in outbox directory
    hf = open(outboxdir+'/'+g+'/'+hostfn,'w' )
    # fill hostfile with group ip's, 127.0.0.1 for master
    for wid in group.keys():
        hf.write(group[wid]['ip']+'\n')
    hf.close()
    
    # open config file for writing in outboxdir
    configfn = 'config'+g
    cf = open(outboxdir+'/'+configfn,'w' )
    cf.write(g+'/'+hostfn+'\n')
    cf.write(g+'/'+execfn+'\n')
    cf.write(job['out']+'\n')
    cf.write(str(job['n'])+'\n')
    cf.close()
    
    return None

# adds new job group to connect.data
def addGroup(group):
    global pool
    global altered
    # members records need to be replaced
    for key in group.keys():
        pool[key] = group[key]
        altered += [key]
    return None

# write pool to connect.data
def writePool(altered=[]):
    global pool
    
    cd = open('connect.data', 'w')
    for k in pool.keys():
        if k not in altered:
            cd.write(pool[k]['ip']+' ')
            cd.write(pool[k]['port']+' ')
            cd.write(pool[k]['status']+' ')
            cd.write(str(pool[k]['gid'])+' ')
            cd.write(str(pool[k]['master'])+'\n')
        
    cd.write('---\n')
    for key in altered:
        # write record
        cd.write(pool[key]['ip']+' ')
        cd.write(pool[key]['port']+' ')
        cd.write(pool[key]['status']+' ')
        cd.write(str(pool[key]['gid'])+' ')
        cd.write(str(pool[key]['master'])+'\n')
        
    return None
    

# grabs the inbox from gmail account - will only see messages with new subject >:(
def grabEmails():
    global emailuser
    global emailpass
    global msgs
    job  = {}
    s = getEmailFeed(emailuser, emailpass)

    xmlstr = minidom.parseString(s)

    n      = int(getText(xmlstr.getElementsByTagName('fullcount')[0].childNodes))

    titles = [getText(i.childNodes) for i in xmlstr.getElementsByTagName('title')[1:]]
    bodies = [getText(i.childNodes) for i in xmlstr.getElementsByTagName('summary')]
    ids    = [getText(i.childNodes).split(':')[2] for i in xmlstr.getElementsByTagName('id')]
    emails = [getText(i.childNodes) for i in xmlstr.getElementsByTagName('email')]
    
    seen = open('.seen', 'a')
    
    for i in range(n):
        if isEmailNew(ids[i]):
            msgs[ids[i]] = [titles[i], bodies[i], emails[i]]
            seen.write(ids[i]+'\n')
    seen.close()
    
    return 
    



# send email to user, informing them where to find output
def emailUser(address='', subject='', body=''):
    global emailuser
    global emailpass
    global jobowners
    
    to = address

    smtpserver = smtplib.SMTP("smtp.gmail.com",587)
    smtpserver.ehlo()
    smtpserver.starttls()
    smtpserver.ehlo
    smtpserver.login(emailuser, emailpass)
    header  = 'To:' + to + '\n' + 'From: ' + emailuser + '\n' + 'Subject:'+subject+' \n'
    print 'Emailing '+to+' regarding job '+str(gid)+' ... ', 
    msg     = header + '\n '+body+' \n\n'
    
    smtpserver.sendmail(emailuser, to, msg)
    print 'Done.'
    smtpserver.close()
    return


# append any new email jobs to jobfile
def submitNewEmailJobs():
    global msgs
    global jobfile
    f = open(jobfile, 'a')
    for k in msgs.keys():
        f.write(msgs[k][1]+','+msgs[k][2]+'\n')
        emailUser(msgs[k][2], 'We have received your job submission.', 'Thank you for submitting a csi702job. You have requested '+msgs[k][1].split(',')[0]+' nodes. You will receive an email when it is ready to be executed.')
        del msgs[k]
    f.close()
    return
    
# returns whether an email id is new
def isEmailNew(s):
    # check .seen for this msg id
    seen = open('.seen', 'r')
    mtmp = seen.read()
    seen.close()
    if mtmp.find(s) == -1:
        return True
    else:
        return False
##########################################################
##########################################################
##########################################################
pool      = {}  # pool of workers
altered   = []  # list of wid keys that have been altered
jobowners = {}  # gid owner emails
jobs      = {}  # dict of current jobs
msgs      = {}  # list of msgs (will be populated by archiveEmails()

# create empty requests file
open('requests', 'w').close()

# if inboxdir and outboxdir don't exist, mk them
if not os.path.exists(inboxdir):
    os.mkdir(inboxdir)
if not os.path.exists(outboxdir):
    os.mkdir(outboxdir)

while True:
    # wait until we can use connect.data
    while isLocked():
        time.sleep(1.0)
        
    # lock connect.data
    changeLock('lock')
    
    # update our internal pool record
    updatePool()
    
    # see if we can add a job from email
    grabEmails()
    submitNewEmailJobs()
    time.sleep(0.5)
    
    # check jobfile for new job request
    job = popJob()
    
    if job:
        # try to recruit group of correct size
        group = recruitGroup(job['n'], job)
        if not group:
            print 'Unable to recruit group of size', job['n']
            pushJob(job)
        else:
            print 'Group generated:\n', group
        
            # generate files for job
            generateJobFiles(group, job)
            
            # update pool to reflect job
            addGroup(group)
            
            # notify user that their job has been added
            emailUser(jobowners[gid-1], 'Your csi702job has begun.', 'Thank you for submitting your csi702 job. It is ready for execution. You will receive an email when it is complete.')
            
    
    # get finished nodes
    group = getFinished()
    
    # for each finished job
    for k in group:
        fingid = group[k]['gid']
        # make sure output is in their subdir
        # with standard name output<GID>
        try:
            os.rename(jobs[fingid]['out'], outboxdir+'/'+str(fingid)+'/'+'output'+str(fingid))
            # notify user that their job is ready for pickup
            emailUser(jobowners[fingid], 'Your csi702project job has completed.','This is a message from the csi702 job server. Your job, #'+str(fingid)+', has finished.\nYour output file is:'+outboxdir+'/'+str(fingid)+'/output'+str(fingid) )
        except OSError:
            print 'Output file does not exist or could not be opened.'
            
        # set finished node to idle and not master
        pool[k]['status'] = 'i'
        pool[k]['master'] = 0
        # remove email address from jobowners
        del jobowners[fingid]
        
    # write current pool to connect.data
    writePool(altered)
        
    # get a fresh copy of the pool
    updatePool()
    
    # free connect.data
    changeLock('unlock')
    
    print 'Sleeping for '+str(sleeptime)+' seconds...'
    time.sleep(sleeptime)
    
    
    
